{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_path = \".\"\n",
    "wiki = load_learner(wiki_path, \"wiki_heb_spm.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "path_r = \"data/rivlin/\"\n",
    "data_frame = {}\n",
    "for i in [\"train\", \"test\"]:\n",
    "    data = list(codecs.open(f\"{path_r}/token_{i}.tsv\", 'r', 'utf-8').readlines())\n",
    "    x, y = zip(*[d.strip().split('\\t') for d in data])\n",
    "    data_frame[i] = pd.DataFrame({\"text\": x, \"label\": y, \"valid\": i==\"test\"})\n",
    "\n",
    "# if you want to be carefull\n",
    "# msk = np.random.rand(len(data_frame[\"train\"])) < 0.8\n",
    "# train = data_frame[\"train\"][msk]\n",
    "# valid = data_frame[\"train\"][~msk]\n",
    "# len(train), len(valid), len(data_frame[\"test\"])\n",
    "\n",
    "dataframe = pd.concat([data_frame[\"train\"], data_frame[\"test\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai.text.data.SPProcessor at 0x7f01dc7c8a90>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spm = SPProcessor(lang=\"heb\", vocab_sz=5000)\n",
    "spm.load(\"spm.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = (TextList.from_df(dataframe, cols=[\"text\"], processor=spm)\n",
    "            .random_split_by_pct(0.2)\n",
    "            .label_for_lm()           \n",
    "            .databunch(bs=36))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>6.329404</td>\n",
       "      <td>5.859082</td>\n",
       "      <td>0.126282</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>5.486599</td>\n",
       "      <td>5.289496</td>\n",
       "      <td>0.163055</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5.015331</td>\n",
       "      <td>4.990203</td>\n",
       "      <td>0.186040</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.483607</td>\n",
       "      <td>4.743812</td>\n",
       "      <td>0.209259</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.074611</td>\n",
       "      <td>4.677798</td>\n",
       "      <td>0.217989</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = language_model_learner(data_lm, AWD_LSTM,pretrained=wiki, drop_mult=0.3)\n",
    "learn.fit_one_cycle(5, 5e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.903867</td>\n",
       "      <td>4.669200</td>\n",
       "      <td>0.222833</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.811990</td>\n",
       "      <td>4.591671</td>\n",
       "      <td>0.231207</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.717663</td>\n",
       "      <td>4.571652</td>\n",
       "      <td>0.235185</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.595308</td>\n",
       "      <td>4.573686</td>\n",
       "      <td>0.236396</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(4, slice(1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder(\"encoder_rivlin_spm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>\" שמע ישראל , השם ישמור ויקרא הגורל = ( י.ק.ו....</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>איחולי הצלחה בתפקידך .</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>כל הכבוד !!!</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>\" בוקר טוב ישראל בוקר טוב לכבוד נשיא מדינת ישר...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>איפה הגינוי ? http://www.iba.org.il/bet/bet.as...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2555</td>\n",
       "      <td>https://www.facebook.com/photo.php?fbid=454613...</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2556</td>\n",
       "      <td>שכל עם ישראל יהיה גאה בך ובמשפחתך הנהדרת .</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2557</td>\n",
       "      <td>מה קרה נהיית נשיא נהיית פרס ? לך תתעסק בהסתות ...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2558</td>\n",
       "      <td>כבוד הנשיא הנבחר . התרגשתי אתמול מנאומך בעצרת ...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2559</td>\n",
       "      <td>הדמעות זולגות מעצמן</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2560 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text label  valid\n",
       "0     \" שמע ישראל , השם ישמור ויקרא הגורל = ( י.ק.ו....     0  False\n",
       "1                                איחולי הצלחה בתפקידך .     0  False\n",
       "2                                          כל הכבוד !!!     0  False\n",
       "3     \" בוקר טוב ישראל בוקר טוב לכבוד נשיא מדינת ישר...     0  False\n",
       "4     איפה הגינוי ? http://www.iba.org.il/bet/bet.as...     1  False\n",
       "...                                                 ...   ...    ...\n",
       "2555  https://www.facebook.com/photo.php?fbid=454613...     2  False\n",
       "2556         שכל עם ישראל יהיה גאה בך ובמשפחתך הנהדרת .     0  False\n",
       "2557  מה קרה נהיית נשיא נהיית פרס ? לך תתעסק בהסתות ...     1  False\n",
       "2558  כבוד הנשיא הנבחר . התרגשתי אתמול מנאומך בעצרת ...     0  False\n",
       "2559                                הדמעות זולגות מעצמן     0  False\n",
       "\n",
       "[2560 rows x 3 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_frame[\"test\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_class = (TextList.from_df(dataframe, processor=spm)\n",
    "            .split_from_df(col = \"valid\")\n",
    "            .label_from_df(cols = \"label\")           \n",
    "            .databunch(bs=36))\n",
    "\n",
    "learn_class = text_classifier_learner(data_class, AWD_LSTM, drop_mult=0.65)\n",
    "learn_class.load_encoder(\"encoder_rivlin_spm\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xVVbbA8d9K7wVSKKF3UGpABQvOKGLFOoplUCzjjDrqlDfzxnnWcXRmfM/eULGOOKNjHSsyIiKihCJdupIESGghMT1Z7497ghe4KUDOPfcm6/v53E/u3fuUdQjJyj57n71FVTHGGGP2F+F1AMYYY0KTJQhjjDEBWYIwxhgTkCUIY4wxAVmCMMYYE1CU1wG0poyMDO3Zs6fXYRhjTNhYuHDhdlXNDFTXphJEz549ycvL8zoMY4wJGyLybWN1dovJGGNMQK4mCBGZLiJFIrK8kfpJIrJURJaISJ6IHOtXN0VE1jqvKW7GaYwx5kButyCeAyY2UT8LGKaqw4GpwNMAItIBuA04ChgD3CYi6e6Gaowxxp+rCUJV5wA7m6gv0x/m+kgEGt6fAsxU1Z2quguYSdOJxhhjTCvzvA9CRM4RkdXAu/haEQBdgc1+m+U7ZYH2v8a5PZVXXFzsbrDGGNOOeJ4gVPUNVR0InA3c5RRLoE0b2X+aquaqam5mZsCRWsYYYw6B5wmigXM7qo+IZOBrMXTzq84BCj0JzBhj2ilPE4SI9BURcd6PBGKAHcCHwAQRSXc6pyc4ZcYYY/zMXLmNJz5d78qxXX1QTkRmAOOBDBHJxzcyKRpAVZ8AzgN+KiI1QAVwodNpvVNE7gIWOIe6U1Ub7ew2xpj26sMVW5m3bjvXntCn1Y/taoJQ1cnN1P8F+EsjddOB6W7EZYwxbUVxaRUZybGuHDtk+iCMMcYcvOLSKjKTLEEYY4zZT3FZFZnWgjDGGOOvrl7Z+X21JQhjjDH72lVeTV29WoIwxhizr+LSKgAyrA/CGGOMv4YEYS0IY4wx+9ibIKwFYYwxxt/2MmtBGGOMCaC4tIr46EgSY9155tkShDHGhCk3n4EASxDGGBO2ikstQRhjjAlge5l702yAJQhjjAlb1oIwxhhzgOraenaV17j2kBxYgjDGmLC043t3h7iCJQhjjAlLbj9FDZYgjDEmLLn9kBxYgjDGmLAU1i0IEZkuIkUisryR+ktEZKnzmiciw/zqNonIMhFZIiJ5bsVojDHhqiFBdEyMce0cbrYgngMmNlG/EThBVYcCdwHT9qs/UVWHq2quS/EZY0zYKi6tIiUuirjoSNfO4c4EHoCqzhGRnk3Uz/P7OB/IcSsWY4xpa9yeZgNCpw/iSuB9v88KfCQiC0XkmqZ2FJFrRCRPRPKKi4tdDdIYY0LF9lL3lhpt4HmCEJET8SWI3/kVj1PVkcCpwHUicnxj+6vqNFXNVdXczMxMl6M1xpjQUFxW5epDcuBxghCRocDTwCRV3dFQrqqFztci4A1gjDcRGmNMaHJ7mg3wMEGISHfgdeAyVV3jV54oIskN74EJQMCRUMYY0x6VV9dSVlXreoJwrZNaRGYA44EMEckHbgOiAVT1CeBWoCPwmIgA1DojlrKBN5yyKOBlVf3ArTiNMSbcbC+tBtxbarSBm6OYJjdTfxVwVYDyDcCwA/cwxhgDvv4HcPchOQiBTmpjjDEHp+EhuTbdSW2MMebgNbQgsqwFYYwxxl9xaRUi0MHFaTbAEoQxxoSd4tIqOibGEBXp7q9wSxDGGBNmtgfhITmwBGGMMWEnGA/JgSUIY4wJO8WlVa4/AwGWIIwxJqyoalBmcgVLEMYYE1b2VNZSXVtvCcIYY8y+Gtaitk5qY4wx+wjGWtQNLEEYY0wYsQRhjDEmoGDNwwSWIIwxJqxsKakgNiqC9IRo189lCcIYY8JIYUklXdLicdbMcZUlCGOMCSNbdlfQOTUuKOeyBGGMMWGkcHclnVPjg3IuSxDGGBMmauvqKSqtpGtamLcgRGS6iBSJyPJG6i8RkaXOa56IDPOrmygi34jIOhH5vVsxGmNMONlWWkW9Que08G9BPAdMbKJ+I3CCqg4F7gKmAYhIJPAocCowGJgsIoNdjNMYY8JC4e4KALqEe4JQ1TnAzibq56nqLufjfCDHeT8GWKeqG1S1GngFmORWnMYYEy72Joh21kl9JfC+874rsNmvLt8pC0hErhGRPBHJKy4udjFEY4zx1paSSqBt3GJqERE5EV+C+F1DUYDNtLH9VXWaquaqam5mZqYbIRpjTEgo3F1BSlwUSbFRQTlfcM7SCBEZCjwNnKqqO5zifKCb32Y5QGGwYzPGmFBTuLsyaP0P4GELQkS6A68Dl6nqGr+qBUA/EeklIjHARcDbXsRojDGhZEtJ8B6SAxdbECIyAxgPZIhIPnAbEA2gqk8AtwIdgcecR8ZrnVtFtSJyPfAhEAlMV9UVbsVpjDHhonB3BcO7pQXtfK4lCFWd3Ez9VcBVjdS9B7znRlzGGBOOKqrr2FVe0z5uMRljjGm5whLfENdg3mKyBGGMMWFgy27fEFdrQRhjjNlHQwuiS5Am6gNLEMYYExYanqLOTnV/JbkGliCMMSYMbNldSWZyLLFRkUE7pyUIY4wJA4UlFUGbg6mBJQhjjAkDhbsrgrZQUIN2nyCqauv4xd8X8sHyrV6HYowxAakqW0qCO80GWIKgts73D3/DjEV8srrI63CMMeYAeypqKa+uo0uQVpJr0O4TRGJsFM9dMYYBnZL52UsL+Xzddq9DMsaYfRQEeaGgBu0+QQCkxkfz4tSj6NUxkauez+OrjY2uc0RNXT35u8qDGJ0xpr3b4sFT1GAJYq/0xBheuuooOqfFMfW5BTw/bxMlFTV76+vrlXe+LuTk//uU4/76CR+v3OZhtMaY9qSwJPhPUYMliH1kJsfy8lVH0zcridveXsFRf/6YX/1zCa8tzOfMR+Zyw4zFxEZFMiA7mZv+sYR1RaVeh2yMaQcKd1cQHSlkJgXvITnweMGgUNQpNY43rxvHsvwSZiz4jreXFPL6ogJy0uO5/8JhnDWsK1v3VDLpkblc/cJC3rxuHKnx0V6HbYxpw7bsriA7JY6IiEALbrrHEkQjjsxJ5cicI7nltEEsLyhhRPd0YqJ8Da6uafE8dskoLn5qPje+sphnpowmMsjfOGNM+1FYUhnUOZgaWIJoRmJsFEf17nhA+ZheHbhj0hBueWM5v/j7QhJjo8jfVUHBrgo6pcbxh9MGMqpHBw8iNsa0NYW7K8jtkR7081ofxGG45KgeXHlsL2atKmL++h2gMLpnOvm7yjnv8S+46ZXFbHU6l4wx5lDU1yvb9lTSOcgd1GAtiMP2P2cM5pbTBu1zb/D7qloem72Opz7byEcrt/GH0wZx6dE9PIzSGBOutpdVUVOnQZ+HCVxsQYjIdBEpEpHljdQPFJEvRKRKRH6zX90mEVkmIktEJM+tGFvL/h1HibFR/PaUgXx88wnk9uzAH99czv9+9A2qus92qsrS/N0s2LSTZfklrN1WSnFpVTBDN8aEOK8ekgN3WxDPAY8ALzRSvxP4JXB2I/UnqmpYP9bcvWMC06fkcssby3n4P+vY8X01d006gsgIYeG3u7j3/VUs2LTrgP1u+FFfbj6pf9BHLBhjQk/DH41ZycFvQbiWIFR1joj0bKK+CCgSkdPdiiEUREVGcO95R9IhKYbHZ69nR1kVgvDBiq1kJMVy56Qh9MpIpLKmnoqaOj79ppiH/7OO5QUlPHDRCBtCa0w7V15dB0BibPDWgWgQqn0QCnwkIgo8qarTGttQRK4BrgHo3r17kMI7OCLC7yYOpGNiDH96dxWJMZHcfFJ/rjquF4mx+34LzhzamRHd07jjnRVMemQuj186ioGdkhGx1oQx7VFDgkiICf6v61BNEONUtVBEsoCZIrJaVecE2tBJHtMAcnNzNdA2oeKq43ozplcHuqTFk9HIE5EiwqVH92Bgp2R+/vdFnPrgZ8RERpCaEE16QjTd0hM4cWAWJw3KppMHnVbGmOAqr64FID7GWhAAqGqh87VIRN4AxgABE0S4GZqT1qLtcnt24N0bjuXNJQXs+L6akvIadpVXs2pLKbNWF/HHN5czNCeVEd3SSI2PJsV5ZSbF0jktjs4p8aTER1nLw5gwV7G3BWEJAhFJBCJUtdR5PwG40+OwPJGVEsc1x/fZp0xVWVdUxsxV2/h45TbeXFLInsoaNEDbKSEmksTYKGIiI4iOFBJiorj06B5cNLqbdYAbEybKa+qIjhSiI4P/2JprCUJEZgDjgQwRyQduA6IBVPUJEekE5AEpQL2I3AQMBjKAN5y/fKOAl1X1A7fiDDciQr/sZPplJ/OL8X0B34M0ZdW1lJTXUFRaxdaSSraUVLClpJLy6jpq6uqpqatn0/bv+cMby/hH3mb+NOkIjsxJ9fhqjDHNqaiuIz46+K0HcHcU0+Rm6rcCOQGq9gDDXAmqjYqIEFLiokmJi6Zbh4RGt1NV3lpSyJ/eXcVZj87lsqN78NtTBpAcZyOljAlV5dW1nnRQg0210a6ICGeP6MqsX5/AlGN68tL8b5n4wGfMs1X0jAlZ5dV1nvQ/gCWIdik1PprbzxrCq9eOJSYqgouf/pJb31rO91W1XodmjNlPRXWdJyOYIAQ7qU3wjOqRznu/PI6/ffgNz87byL+XbqFvVhI5afF0TY+nQ2IM/l3ZI7qnM6xby0ZhGWNah5ctCEsQ7Vx8TCS3njmYiUd0YsZX35G/q5z5G3awdU8l9fuNjIoQ+NXJ/fnF+L42CsqYICmvriU1IcaTc1uCMIBvfYsxvX5Yv6Kmrp6yyh9uOVXX1fPn91Zx30dr+GrTLu7/yTA6Bnn5Q2Pao/LqOjqnWh+ECSHRkRGkJ8bsfWWnxPHAhcO5+5wjmL9hB6c/NJcFm3a2+nlr6+p5af63vDj/Wypr6lr9+MaEm/LqOhI8mIcJLEGYgyAiXHJUD17/+VjioiO4aNp8Hv1kHfX734s6RGu2lXLu4/P445vL+Z83l/Pj//2UV/M2U9dKxzcmHFXUWB+ECSNHdE3lnRuO5b9fX8bfPvyG+Rt2cP+Fw+mYGMP64jLmrt3O0oIS+mQmkdvD17Ed18SDPrV19Tw5ZwMPfryWpLgoHrl4BOkJMfz1g9X89rWlPDlnAxfmdmN49zSO6JLq2YgOY7zg5XMQLTqriPQB8lW1SkTGA0OBF1R1t5vBmdCVHBfNw5NHMLZPBne8s4KJD8whKiKCrXt8S6x2TIzh9UUFAERHCoO7pNIvK4nemYn0zkgkLjqSJZt3s/DbXSz5bjelVbWcfmRn7pg0ZO9Ehm9eN44PV2zl/plrufu9VQBERgiDOidz/Yn9mHhEJ28u3pggqa9XKmvqQ/5J6n8BuSLSF3gGeBt4GTjNrcBM6BMRLj6qOyO6p3H3u6tITYjm2L4ZjOuTQfeOCewur2bht7tYsGkXSzbvYs6aYl5bmO+3PwzITuas4V04aVA2Jw7MOuD4E4/ozMQjOlNcWsWSzbtZsnkXH68s4tqXFvLTY3rwh9MGNdk6MSacVdR4N1EftDxB1KtqrYicAzygqg+LyGI3AzPhY1DnFF666qgDytMSYvjxoGx+PCh7b1lZVS0bi7+nrKqWI7qmtHiaj8zkWE4enM3Jg7O58cf9+esHq3l67kYWbNrFIxePoE9mUsD9tpf55qYa0iXFZrY1Yafcw5lcoeUJokZEJgNTgDOdMpvAxxy0pNiow54kMCYqgj+eMZixfTvy639+zZkPz+Xq43oz9dhee1fgq69X/v7Vd/z1g9WUVtZyXL8Mbj1jMP2yk1vjMowJioapvuNDuQ8CuAK4FrhbVTeKSC/gJffCMqZ5PxqYzfs3Hs/tb6/gwVlrefbzjVx1XG/G9unIn95dxZLNuxnbpyPH9svgidnrmfjgZ1x2dA8uPboHtfX1lFfXUVldR9/sJE/W+zWmOeU1vmeRvGpBiAZaSKCpHUTSgW6qutSdkA5dbm6u5uXleR2G8cCKwhIe+HgtM1duA3yd5H88YxBnD++KiLCjrIr/m7mGGV99d8AT4vHRkVx7Qh+uOb63jZAyIWXRd7s497F5PHvFaE4ckNX8DodARBaqam6gupaOYpoNnOVsvwQoFpFPVfVXrRalMYdhSJdUnvppLsvyS8j7difnjOhKmt/0BB2TYrn7nCOZMrYnS/NLSIiJJD4mkuiICGZ89R33f7yGVxZ8x+9PHciZQ7vYVCImJOxdTS7ERzGlquoeEbkKeFZVbxORkGtBGHNkTmqTfRz9s5Ppv18/xLH9MpiycSd3/nsFN76yhFvfWsHwbmmM7J7OyB5pHN27oyereRnzQyd1aPdBRIlIZ+AnwC0uxmOMJ8b06sDb1x3L+8u3MnddMYu+3c0Ds9agCp1T47h8bE8uGtN9bye4McFQXu3rgwj16b7vBD4EPlfVBSLSG1jrXljGBF9EhHD60M6cPrQzAHsqa5i/fgfPzdvEPe+v5qFZazlreFdioyLYU1FDSUUNERHCxWO6M35Apg2jNa0uLIa5quqrwKt+nzcA5zW1j4hMB84AilT1iAD1A4FngZHALap6n1/dROBBIBJ4WlXvbUmcxrSmlLhoJgzpxIQhnVheUML0uRv516J8YiMjSE2IJjU+mu1lVcxcuY2BnZL5+fg+nDQom7VFZawoLGF5wR6KSyuprVfq6pXaOmVot1SuOrY3mck2E65pXkOCSPToFlOLRjGJSA7wMDAOUGAucKOq5jexz/FAGb4pOQIliCygB3A2sKshQYhIJLAGOBnIBxYAk1V1ZXNx2igm4zZV3aelUFNXz1tLCnl89jrWF3+/z7YpcVHkpCcQHSlERgj1Ckvzd/tW8RvTg5+d0JvsFBteaxr3yH/Wct9Ha1jzp1OJiXKnH+ywRzHh+0v/ZeAC5/OlTtnJje2gqnNEpGcT9UVAkYicvl/VGGCd00pBRF4BJgHNJghj3Lb/baToyAjOH5XDuSO6MnPVNlYW7mFQ52SGdEklJz3+gO03FJfx2Oz1PP/FJl768luuG9+X607sQ5R1gpsAyqvriIoQ15JDc1p61kxVfVZVa53Xc0CmSzF1BTb7fc53ygISkWtEJE9E8oqLi10KyZimRUQIpwzpxM0n92fiEZ3p1iEhYJ9E78wk7rtgGJ/8ejwTh3Ti/o/XcPFTX1K4u8KDqE2oK/dwPWpoeYLYLiKXikik87oU2OFSTIF6+hq9D6aq01Q1V1VzMzPdylnGtK7uHRN4aPII7r9wGCsKSzj1wc/4YPkWr8MyIabCw/WooeUJYiq+Ia5bgS3A+fim33BDPtDN73MOUOjSuYzx1Dkjcnj3l8fRo2MC1760iOlzN3odkgkh5TV1nj0DAS1MEKr6naqepaqZqpqlqmcD57oU0wKgn4j0EpEY4CJ804sb0yb1zEjktWvHMnFIJ+56d6W1JMxeFdW1nq0FAYe35GiT02yIyAzgC2CAiOSLyJUicq2IXOvUdxKRfOc4f3S2SVHVWuB6fM9drAL+qaorDiNOY0JeTFQED1w0nOHd0rjxlSUs/HaX1yGZEFDu8S2mw2m7NPlUkKpObqZ+K77bR4Hq3gPeO/TQjAk/cdGRPP3TXM59fB5Xv5DH6z8fS8+MRK/DMh4qr64jOc67W0yHc2ZbSd6YVtYxKZbnrhjDuY99ziVPf8mJAzNJi48hLSGaPllJrs3oaUJTRXUd2SnePVTZZIIQkVICJwIB4l2JyJh2rldGIk9PGc0tbyzj3aVbKKmo2TtF+R1nDWHK2J6exmeCp7ym1tNO6ibPrKq2/JYxHhjVI50Pbjoe8K2OV1pZy69f/Zo73llBTnr8Psu4mrarvCo8noMwxngkIkJITYjmocnDGdIlletfXsyy/BKvwzJBUF5dR6IlCGNMcxJionjm8lw6JMYw9fkFFNjT121afb1SUVPn2XrUYAnCmLCSlRzHs1eMprK6jnMf+5x73l/F15t3c7BLB5vQV1nr7VTfYAnCmLDTPzuZ56aOYUCnFJ75bCOTHv2cY//yCY/PXk/d/gtum7Dl9VoQcHjDXI0xHhnVI50Xpo5hd3k1H68q4q0lBfzlg9V8traYBy4aTlayTSMe7hrWow7XJ6mNMR5LS4jh/FE5vHjlUfzt/KEs+m4Xpz34GXPXbvc6NHOYvF6PGixBGNNmXJDbjbevP5b0hBgum/4l97y/isqaOq/DMoeoYT1q64MwxrSK/tnJvHX9OC7M7caTn27gtAc/Y8GmnV6HZQ7B3ltMliCMMa0lISaKe88byotXjqG6rp4LnviCW99azvdVtV6HZg5CKHRSW4Iwpo06rl8mH950PJeP7cmL87/lwmlfUFRa6XVYpoXKayxBGGNclBgbxe1nDWH6lNGsL/qecx+bx/riMq/DMi1Q4fRB2INyxhhXnTgwi3/87Ggqa+o47/F55Fm/RMjbe4vJhrkaY9w2NCeN138+jvSEGC55+kvmrCn2OiTThHLrpDbGBFP3jgn86+dj6Z2ZxM9eXMii7w5cua60soYlNn2H58qra4mMEGKjvPs1bQnCmHamQ2IMz08dTVZKLFOfW8CabaV76+au3c4p98/h7Ec/58rn8/h2x/ceRtq+lVfXkRAdiUiTi3e6yrUEISLTRaRIRJY3Ui8i8pCIrBORpSIy0q+uTkSWOK+33YrRmPYqKzmOF6ceRXRkBJc98yVrtpXyP28u59JnviQuJpJf/qgvX27Ywcn3z+H+mWvsgTsPVFR7uxYEuDsX03PAI8ALjdSfCvRzXkcBjztfASpUdbiLsRnT7nXvmMALU8dw4ZNfMOH+OYjAlcf24renDCAuOpJLju7B3e+u4sFZa/nP6iLe+MVYoiLtpkOwlFfXeTrEFVxsQajqHKCpoRKTgBfUZz6QJiKd3YrHGHOgQZ1TePaK0RzfP5MZVx/N/5wxmDhn1Ex2ShwPTR7BfRcMY1lBCa8vKvA42valvNrbtSDA2z6IrsBmv8/5ThlAnIjkich8ETm7qYOIyDXOtnnFxTYqw5iDNapHB16YOoaje3cMWH/eyK4M75bG/R/braZgqqipbbstiBYI1PPSMGyiu6rmAhcDD4hIn8YOoqrTVDVXVXMzMzPdiNOYdk1E+N3EgWwpqeT5eZu8DqfdaNO3mFogH+jm9zkHKARQ1YavG4DZwIhgB2eM+cExfToyfkAmj81eT0lFjdfhtAsV1XWergUB3iaIt4GfOqOZjgZKVHWLiKSLSCyAiGQA44CVHsZpjAH+65SB7Kms4YlP13sdSrsQCi0I13pARGQGMB7IEJF84DYgGkBVnwDeA04D1gHlwBXOroOAJ0WkHl8Cu1dVLUEY47HBXVKYNKwL0+du5MLcbmzYXsYnq4v5YsMOfjwoi99PHOjpmP22JhQ6qV07u6pObqZegesClM8DjnQrLmPMofv1hAG8u2wL4++bDfiWw+zfKZknP91AVU09t5052JJEK6mo9r6T2takNsa0WLcOCdw16Qi+2VbKiQOyGNOrA7FREdz97iqenrsREbj1DEsSh0tVKa9pw7eYjDFt00Vjuh9Qdsvpg6hT5dnPNxEpwi2nD7IkcRiqautR9XaiPrAEYYxpBSLCrWcMRhWenruRLSWV3DFpCBlJsV6HFpYaVv9LbMcPyhlj2hAR4bYzB/PbUwYwc+U2Tvq/T3ljcb7NCnsIQmGqb7AEYYxpRSLCdSf25d1fHkuvjERu/sfXTH1uAYW7K7wOLaxUhMByo2AJwhjjgn7Zybx27VhuPWMw8zfsZML9c/j7l99SX2+tiZbYu5qcJQhjTFsUGSFMPbYXH950PENzUrnljeVc/PR8W2OiBcob1qOOtj4IY0wb1r1jAn+/6ijuOfdIVhTsYeIDn7E4wEp25gcV1oIwxrQXIsLkMd358ObjyUiO4eoXFlJg/RKNsltMxph2p0taPNOnjKaqpo4rn1tAmTOc0+yrwkYxGWPao37ZyTxyyUjWFpVx44zF1FnH9QEa+iAS7DkIY0x7c0L/TG4/czCzVhdxz3urvA4n5JSHyDBXe5LaGOOJy47pyfri73l67kZ6dEzgsmN6eh1SyKiorkMEYqO8/RveEoQxxjN/PH0Qm3eWc9vbK+icGs9Jg7O9DikklFfXkRAd6fl8VnaLyRjjmajICB6+eARDuqRyw4zFLM3f7XVIIaG8uo6EWO//frcEYYzxVEJMFM9cnkuHxBimPpfH5p3lXofkufIQWAsCLEEYY0JAVnIcz08dTXVtHVc+v2DvMM/2qjwE1qMGSxDGmBDRNyuZR53hr7e/vcLrcDxVEQLrUYPLCUJEpotIkYgsb6ReROQhEVknIktFZKRf3RQRWeu8prgZpzEmNBzXL5NfjO/DP/I289aSAq/D8YzvFlPb74N4DpjYRP2pQD/ndQ3wOICIdABuA44CxgC3iUi6q5EaY0LCzSf1J7dHOn94fRmbtrfPif3Kq+s8f4oaXE4QqjoH2NnEJpOAF9RnPpAmIp2BU4CZqrpTVXcBM2k60Rhj2oioyAgenDyCqMgIrp+xiKra9tcfUREC61GD930QXYHNfp/znbLGyg8gIteISJ6I5BUXF7sWqDEmeLqmxXPfBcNYXrCHP7y+nJq6eq9DCqry9tAH0QKBngLRJsoPLFSdpqq5qpqbmZnZqsEZY7xz8uBsbjqpH/9alM8Vzy6gpLzG65CCpqK6zvO1IMD7BJEPdPP7nAMUNlFujGlHbjqpP387fyhfbtzBOY99zsZ20CehqvYchONt4KfOaKajgRJV3QJ8CEwQkXSnc3qCU2aMaWcuyO3Gy1cfze6KGs5+9HPeXboF1bY7A2xVbT316v1U3+D+MNcZwBfAABHJF5ErReRaEbnW2eQ9YAOwDngK+AWAqu4E7gIWOK87nTJjTDs0umcH3rpuHJ1T47ju5UVMevRzPltb3CYTRaisJgcuT9anqpObqVfgukbqpgPT3YjLGBN+unVI4N83HMvriwt48OO1XPbMVxzTuyP3nnckPTomeh1eqwmVqb7B+1tMxhjTYlGREfwktxv/+c0J3HrGYFZu2cNPnvyC9cVlXofWajY415KRFOtxJJYgjDFhKDYqkpkX344AAA/LSURBVKnH9uKfPzuGunrlwifns2ZbqddhtYo3FheQHBvFuL4ZXodiCcIYE74GdErmlWuOIULgomnzWVm4x+uQDkt5dS0fLN/K6UM7E2eT9RljzOHpm5XEP352DLFREUx+an5YT8/x4YqtlFfXce7IHK9DASxBGGPagF4ZifzjGt/tptvfWRG2o5teX1RATno8uT1CY+o5SxDGmDahe8cEbjqpH7O/KWbWqiKvwzlo2/ZU8vm67Zw7oisREd4uNdrAEoQxps2YMrYnfbOSuOPfK6isCa9J/t5aUkC9wjkhcnsJLEEYY9qQ6MgI7jhrCJt3VjBtzgbXztMwHUZren1RASO6p9ErI3Se6fB+NihjjGlF4/pmcPqRnXn0k3WcO7IrOekJgO+Xer1C5CHevvlyww7mrtvOsoISluWXsKu8mscuGcnEIzofdswrC/ewemspd519xGEfqzVZgjDGtDl/OH0Q/1ldxH+9tpQjc1JZUbCH5YUlxEVF8vb148hKiTuo4y3LL+HCafOJjBD6ZSVx4sAsVhbu4df//Jo+mUn0y04+rHhfX5RPdKRwxpGHn2xak91iMsa0OV3T4rnhx32Zt34Hz87dRElFDRMGZ1NSUcMvX1lMXf3BjXJ6deFmYqMiWHDLSXxw0/Hcd8Ewpl8+mviYKK55cSF7Kg99KnJV5a2vC/nRwCzSE2MO+ThusBaEMaZN+vkJfZg4pBM56QnERPn+Fh7TqyO/efVrHpy1ll+d3L9Fx6mqreOtJYWcMqQTHfx+gXdKjeOxS0Zy8VPzufmVJTz109xDGn1UsLuC4tIqju8feuvZWAvCGNMmiQi9M5P2JgeA80flcP6oHB7+z1rmrt3eouPMWlVESUUN5486cHTRmF4duPXMwcxaXcQDs9YeUpxrt/nmXup/mLep3GAJwhjTrtw5aQh9M5O46R+LKdpT2ez2ry3Mp1NKXKNzI112dA/OH5XDQ7PW8mre5oDbNGVtkW8Oqf5ZliCMMcZTCTFRPHbJSL6vquPqF/KaTBJFeyr5dE0x547s2ujoJxHh7nOO4Lh+GfzuX0v5YPnWg4pnzbYyspJjSU2IPqj9gsEShDGm3emXncyDFw1nzbYyTn94Ll9tDLwe2ZtLCqirV84LcHvJX2xUJE9cOoph3dL45YzFfL6uZbevANZuKw3J20tgCcIY005NGNKJN68bR1JsFJOfms8zczfuM4eTqvLawnxGdk+jT2ZSs8dLjI3i2ctH0ysjkatfyGPxd7ua3ae+XllbVEbfrOaP7wVLEMaYdmtAp2Teun4cJw3K4q5/r+Sn079iwSZfa2JZQQlrtpVx/qhuLT5eWkIML145hoykWK55cSHVtfVNbl+wu4Ly6rr22YIQkYki8o2IrBOR3weo7yEis0RkqYjMFpEcv7o6EVnivN52M05jTPuVEhfNE5eO4rYzB7OycA8XPPEFP3niC/5v5hpioyI4fejBPbyWlRLH7WcNpri0itnfND1p4N4O6ux21oIQkUjgUeBUYDAwWUQG77fZfcALqjoUuBO4x6+uQlWHO6+z3IrTGGNEhCvG9WLu737EbWcOZvOucmZ/U8yEIZ1IjT/4zuPj+mWSkRTD64sKmtyuYYjr4T6J7RY3H5QbA6xT1Q0AIvIKMAlY6bfNYOBm5/0nwJsuxmOMMU2Kj4nkinG9uOSoHsxatY1Rh7guQ3RkBGcN68qL8zexu7yatITAT0iv2VZGdkrsISWhYHDzFlNXwH9QcL5T5u9r4Dzn/TlAsoh0dD7HiUieiMwXkbMbO4mIXONsl1dcXNxasRtj2rGYqAhOPbLzQc/Z5O+8UV2pqVPeWbql0W3WFpXSLwSff2jgZoIINGh4/wlQfgOcICKLgROAAqBhDt3uqpoLXAw8ICJ9Ap1EVaepaq6q5mZmht6j6saY9mlw5xQGdkrmXwvzA9bX1ytrt5XRL0T7H8DdBJEP+Hf/5wCF/huoaqGqnquqI4BbnLKShjrn6wZgNjDCxViNMaZViQjnjuzKks27WV9cdkB9we4KKmpCdwQTuJsgFgD9RKSXiMQAFwH7jEYSkQwRaYjhv4HpTnm6iMQ2bAOMY9++C2OMCXlnD+9KhMAbATqrQ30EE7iYIFS1Frge+BBYBfxTVVeIyJ0i0jAqaTzwjYisAbKBu53yQUCeiHyNr/P6XlW1BGGMCStZKXEc1y+TNxYXUL/fFONrnBFMfUO4D8LV6b5V9T3gvf3KbvV7/xrwWoD95gFHuhmbMcYEw7kju3LjK0uYv3EHY/v8MOHfmm2lIT2CCexJamOMcdWEwZ1Iio3itbx9O6vXbisL6f4HsARhjDGuio+J5ILcHN5YUsDCb33TeNTXK+uKykJ6iCtYgjDGGNf9esIAuqTG89vXllJZU0f+roYRTKHbQQ2WIIwxxnVJsVHce96RbCj+nvs/XrN3BFOoTrHRwNakNsaYIDiuXyYXje7GU3M28N2OcoCQnea7gbUgjDEmSP5w+iCyU+J4f/lWOqXEhfQIJrAEYYwxQZMSF8095/pG8IfyFBsN7BaTMcYE0fgBWdx25uAWrVLnNUsQxhgTZFeM6+V1CC1it5iMMcYEZAnCGGNMQJYgjDHGBGQJwhhjTECWIIwxxgRkCcIYY0xAliCMMcYEZAnCGGNMQKKqzW8VJkSkGPh2v+JUoKSZMv/Pgd43fM0Ath9ieIHiaEl9a8YPh34NzcXf1DZNxbv/5+beW/wHv01z/4cau57WjL+p+Jqrt59hd+PvoaqZAfdQ1Tb9AqY1V+b/OdB7v695rRlHS+pbM/7DuYbm4j+YazjY+Fvje2DxN17W2PW0ZvwtuYZg/AxY/Ae3T3u4xfROC8reaeZ9oGO0RhwtqQ+X+Jvapql49//ckveHwuJvvKyx62nN+FtyjHD/GQj3+A/Qpm4xuU1E8lQ11+s4Dke4X4PF7y2L31vBjr89tCBa0zSvA2gF4X4NFr+3LH5vBTV+a0EYY4wJyFoQxhhjArIEYYwxJqB2myBEZLqIFInI8kPYd5SILBORdSLykIiIX90NIvKNiKwQkb+2btT7xNDq8YvI7SJSICJLnNdprR/5PnG48j1w6n8jIioiGa0X8QExuPE9uEtEljr//h+JSJfWj3xvDG7E/zcRWe1cwxsiktb6ke+NwY34L3B+dutFxJXO4MOJu5HjTRGRtc5ril95kz8jLXKoY2rD/QUcD4wElh/Cvl8BxwACvA+c6pSfCHwMxDqfs8Is/tuB34Tz98Cp6wZ8iO+hyYxwih9I8dvml8ATYRb/BCDKef8X4C9hFv8gYAAwG8gNpbidmHruV9YB2OB8TXfepzd1jQfzarctCFWdA+z0LxORPiLygYgsFJHPRGTg/vuJSGd8P8RfqO+78AJwtlP9c+BeVa1yzlEUZvEHlYvXcD/wX4CrIzDciF9V9/htmoiL1+BS/B+paq2z6XwgJ8ziX6Wq37gV8+HE3YhTgJmqulNVdwEzgYmt9XPebhNEI6YBN6jqKOA3wGMBtukK5Pt9znfKAPoDx4nIlyLyqYiMdjXaAx1u/ADXO7cHpotIunuhNuqwrkFEzgIKVPVrtwNtxGF/D0TkbhHZDFwC3OpirIG0xv+hBlPx/eUaTK0ZfzC1JO5AugKb/T43XEurXGPUwe7QVolIEjAWeNXvVl1soE0DlDX8lReFr5l3NDAa+KeI9HYyuKtaKf7Hgbucz3cB/4vvhzwoDvcaRCQBuAXfbY6ga6XvAap6C3CLiPw3cD1wWyuHGlBrxe8c6xagFvh7a8bYlNaMP5iailtErgBudMr6Au+JSDWwUVXPofFraZVrtATxgwhgt6oO9y8UkUhgofPxbXy/RP2bzTlAofM+H3jdSQhfiUg9vsm1it0M3HHY8avqNr/9ngL+7WbAARzuNfQBegFfOz9oOcAiERmjqltdjh1a5/+Qv5eBdwlSgqCV4nc6Ss8AfhyMP478tPa/f7AEjBtAVZ8FngUQkdnA5aq6yW+TfGC83+ccfH0V+bTGNbrRCRMuL6Anfh1FwDzgAue9AMMa2W8BvlZCQ+fPaU75tcCdzvv++Jp+Ekbxd/bb5mbglXD7Huy3zSZc7KR26XvQz2+bG4DXwiz+icBKINPt/ztu/v/BxU7qQ42bxjupN+K7c5HuvO/QkmtsUZzB+CaG4guYAWwBavBl2yvx/fX5AfC185/81kb2zQWWA+uBR/jhifQY4CWnbhHwozCL/0VgGbAU319and2K361r2G+bTbg7ismN78G/nPKl+CZX6xpm8a/D94fREufl5igsN+I/xzlWFbAN+DBU4iZAgnDKpzr/7uuAKw7mZ6S5l021YYwxJiAbxWSMMSYgSxDGGGMCsgRhjDEmIEsQxhhjArIEYYwxJiBLEKZNE5GyIJ/vaREZ3ErHqhPfrK7LReSd5mZGFZE0EflFa5zbGLAV5UwbJyJlqprUiseL0h8mo3OVf+wi8jywRlXvbmL7nsC/VfWIYMRn2j5rQZh2R0QyReRfIrLAeY1zyseIyDwRWex8HeCUXy4ir4rIO8BHIjJeRGaLyGviW/vg7w1z7Tvluc77Mmfiva9FZL6IZDvlfZzPC0Tkzha2cr7ghwkJk0RklogsEt98/5Ocbe4F+jitjr852/7WOc9SEbmjFf8ZTTtgCcK0Rw8C96vqaOA84GmnfDVwvKqOwDeL6p/99jkGmKKqP3I+jwBuAgYDvYFxAc6TCMxX1WHAHOBqv/M/6Jy/2flxnLmEfozv6XaASuAcVR2Jbw2S/3US1O+B9ao6XFV/KyITgH7AGGA4MEpEjm/ufMY0sMn6THt0EjDYb+bMFBFJBlKB50WkH76ZL6P99pmpqv5z+H+lqvkAIrIE39w6c/c7TzU/THi4EDjZeX8MP8zN/zJwXyNxxvsdeyG+uf7BN7fOn51f9vX4WhbZAfaf4LwWO5+T8CWMOY2cz5h9WIIw7VEEcIyqVvgXisjDwCeqeo5zP3+2X/X3+x2jyu99HYF/lmr0h06+xrZpSoWqDheRVHyJ5jrgIXzrRGQCo1S1RkQ2AXEB9hfgHlV98iDPawxgt5hM+/QRvnUWABCRhmmWU4EC5/3lLp5/Pr5bWwAXNbexqpbgW370NyISjS/OIic5nAj0cDYtBZL9dv0QmOqsN4CIdBWRrFa6BtMOWIIwbV2CiOT7vX6F75dtrtNxuxLfNO0AfwXuEZHPgUgXY7oJ+JWIfAV0Bkqa20FVF+Ob6fMifIvw5IpIHr7WxGpnmx3A586w2L+p6kf4bmF9ISLLgNfYN4EY0yQb5mpMkDkr31WoqorIRcBkVZ3U3H7GBJv1QRgTfKOAR5yRR7sJ4rKuxhwMa0EYY4wJyPogjDHGBGQJwhhjTECWIIwxxgRkCcIYY0xAliCMMcYE9P9y9dtVk6wipwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_class.lr_find()\n",
    "learn_class.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.625178</td>\n",
       "      <td>0.520488</td>\n",
       "      <td>0.776953</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.599120</td>\n",
       "      <td>0.471682</td>\n",
       "      <td>0.797266</td>\n",
       "      <td>00:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.578954</td>\n",
       "      <td>0.474277</td>\n",
       "      <td>0.803906</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_class.fit_one_cycle(3, 1e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.585205</td>\n",
       "      <td>0.447977</td>\n",
       "      <td>0.815625</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.556424</td>\n",
       "      <td>0.454897</td>\n",
       "      <td>0.808594</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.549100</td>\n",
       "      <td>0.457875</td>\n",
       "      <td>0.805469</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_class.freeze_to(-2)\n",
    "learn_class.fit_one_cycle(3, slice(1e-4,1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.516384</td>\n",
       "      <td>0.421427</td>\n",
       "      <td>0.828125</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.476250</td>\n",
       "      <td>0.385576</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.436659</td>\n",
       "      <td>0.373535</td>\n",
       "      <td>0.858203</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_class.freeze_to(-3)\n",
    "learn_class.fit_one_cycle(3, slice(1e-4,1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.464614</td>\n",
       "      <td>0.366079</td>\n",
       "      <td>0.856250</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.463398</td>\n",
       "      <td>0.360050</td>\n",
       "      <td>0.860937</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.411934</td>\n",
       "      <td>0.350257</td>\n",
       "      <td>0.864062</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.441111</td>\n",
       "      <td>0.339395</td>\n",
       "      <td>0.873047</td>\n",
       "      <td>00:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.374554</td>\n",
       "      <td>0.317704</td>\n",
       "      <td>0.880859</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.365729</td>\n",
       "      <td>0.309121</td>\n",
       "      <td>0.882031</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.336927</td>\n",
       "      <td>0.301888</td>\n",
       "      <td>0.884375</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.335863</td>\n",
       "      <td>0.303164</td>\n",
       "      <td>0.888281</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.317512</td>\n",
       "      <td>0.297304</td>\n",
       "      <td>0.889063</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.319974</td>\n",
       "      <td>0.297859</td>\n",
       "      <td>0.891406</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_class.unfreeze()\n",
    "learn_class.fit_one_cycle(10, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='10' class='' max='20', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      50.00% [10/20 03:06<03:06]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.335788</td>\n",
       "      <td>0.312303</td>\n",
       "      <td>0.887500</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.336488</td>\n",
       "      <td>0.298386</td>\n",
       "      <td>0.883984</td>\n",
       "      <td>00:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.343671</td>\n",
       "      <td>0.296151</td>\n",
       "      <td>0.889844</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.338922</td>\n",
       "      <td>0.300073</td>\n",
       "      <td>0.889844</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.319843</td>\n",
       "      <td>0.295026</td>\n",
       "      <td>0.892578</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.324818</td>\n",
       "      <td>0.292074</td>\n",
       "      <td>0.896094</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.322001</td>\n",
       "      <td>0.301220</td>\n",
       "      <td>0.895312</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.292184</td>\n",
       "      <td>0.293781</td>\n",
       "      <td>0.894531</td>\n",
       "      <td>00:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.341255</td>\n",
       "      <td>0.292937</td>\n",
       "      <td>0.895703</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.291389</td>\n",
       "      <td>0.296579</td>\n",
       "      <td>0.895312</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='284', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-fbf8ad30fd08>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1e-4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    258\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0mbs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 260\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mreset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/models/awd_lstm.py\u001b[0m in \u001b[0;36mreset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnns\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqrnn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mLinearDecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/models/awd_lstm.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnns\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqrnn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_one_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mLinearDecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/models/awd_lstm.py\u001b[0m in \u001b[0;36m_one_hidden\u001b[0;34m(self, l)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0;34m\"Return one hidden state.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m         \u001b[0mnh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_hid\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0ml\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memb_sz\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m//\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_dir\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mone_param\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mselect_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn_class.fit_one_cycle(20, slice(1e-5,1e-4), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_class.fit_one_cycle(20, slice(1e-6,1e-4), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='5' class='' max='20', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      25.00% [5/20 01:28<04:24]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.155187</td>\n",
       "      <td>0.275143</td>\n",
       "      <td>0.927734</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.156759</td>\n",
       "      <td>0.278164</td>\n",
       "      <td>0.928906</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.132721</td>\n",
       "      <td>0.270799</td>\n",
       "      <td>0.926562</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.126769</td>\n",
       "      <td>0.274131</td>\n",
       "      <td>0.920313</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.155931</td>\n",
       "      <td>0.270577</td>\n",
       "      <td>0.923047</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='284', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-374:\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/anaconda3/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/opt/anaconda3/lib/python3.7/threading.py\", line 870, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/pin_memory.py\", line 21, in _pin_memory_loop\n",
      "    r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)\n",
      "  File \"/opt/anaconda3/lib/python3.7/multiprocessing/queues.py\", line 113, in get\n",
      "    return _ForkingPickler.loads(res)\n",
      "  File \"/opt/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/reductions.py\", line 284, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/opt/anaconda3/lib/python3.7/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/opt/anaconda3/lib/python3.7/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/opt/anaconda3/lib/python3.7/multiprocessing/connection.py\", line 492, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/opt/anaconda3/lib/python3.7/multiprocessing/connection.py\", line 619, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-65-cb526628bc1c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1e-6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    241\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmasked_concat_pool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 243\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    244\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1367\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1368\u001b[0m         \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1369\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1370\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1371\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn_class.fit_one_cycle(20, slice(1e-6,1e-5), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_class.fit_one_cycle(20, slice(1e-7,1e-5), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# p = learn_class.get_preds(ds_type = DatasetType.Test, ordered=True)\n",
    "# results = list(map(int, data_frame[\"test\"][\"label\"].values)) == np.argmax(p[0].numpy(), axis=1)\n",
    "# results.sum() / len(p[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
